using System;
using System.Collections.Generic;
using System.Text;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;
using System.Reflection;
using System.Threading;

using Jeffrey.XThreading;
using Jeffrey.XFramework.XManager;
using Jeffrey.XFramework.XNet;
using Jeffrey.XFramework.XCrawl;
using Jeffrey.XFramework.XInteract;

namespace Jeffrey.XFramework.XCrawl
{
    public class CrawlController
    {
        private static readonly Object s_SyncRoot = new Object();

        //Current workspace path...
        public static readonly String s_Directory_Path;

        static CrawlController()
        {
            s_Directory_Path = Directory.GetCurrentDirectory();
        }

        public CrawlController(CrawlOrder order, UriContainer uc)
            :this(order, uc, null)
        { }

        public CrawlController(CrawlOrder order, UriContainer uc, UriContainer prioruc)
        {
            m_CrawlOrder = order;
            m_IsRunning = false;

            m_CrawlUriManagement = new UriManagement(uc, prioruc);
            m_RequestCollection  = new Dictionary<Int32, XWebRequest>();
            m_Filters            = new List<Filter>();
            m_SpiderXisRunning   = new List<Boolean>();

            m_ProcessChain   = null;
            m_PreFetchChain  = new ProcessorChain(ProcessorChain.ProcessorChainName.PreFetchChain);
            m_FetchChain     = new ProcessorChain(ProcessorChain.ProcessorChainName.FetchChain);
            m_PostFetchChain = new ProcessorChain(ProcessorChain.ProcessorChainName.PostFetchChain);
            m_ExtractChain   = new ProcessorChain(ProcessorChain.ProcessorChainName.ExtractChain);
            m_StoreChain     = new ProcessorChain(ProcessorChain.ProcessorChainName.StoreChain);

            m_Xbject = new ObservableXubject();

            m_Timer = new System.Timers.Timer(1000); // 1s
        }

        public Boolean Initialize()
        {
            m_Timer.AutoReset = true;
            m_Timer.Enabled = true;
            m_Timer.Elapsed += CheckIfAllStops;

            //linked the chain loop...
            m_ProcessChain = m_PreFetchChain;

            m_PreFetchChain.NextProcessorChain  = m_FetchChain;
            m_FetchChain.NextProcessorChain     = m_PostFetchChain;
            m_PostFetchChain.NextProcessorChain = m_ExtractChain;
            m_ExtractChain.NextProcessorChain   = m_StoreChain;

            if (!m_CrawlOrder.Initialize() || !m_CrawlUriManagement.Initialize())
                return false;

            CreateProcessorChain();

            if (!Logger.Initialize(this,
                    CrawlToolKits.GetAbsolutePath(s_Directory_Path, m_CrawlOrder.m_LoggerPath))
                )
                return false;

            XThreadPool.Initialize();

            m_MaxThreadNum = XThreadPool.MaxNum;

            lock (s_SyncRoot)
            {
                for (Int32 i = 0; i < m_MaxThreadNum; i++)
                    m_SpiderXisRunning.Add(false);
            }

            m_CrawlOrder.ReleaseProcessorInfo();

            return true;
        }

        public Boolean LoadConfig()
        {
            if (m_CrawlOrder.LoadConfigFile() && CreateProcessorChain())
                return true;

            return false;
        }

        public Boolean SaveConfig()
        {
            return m_CrawlOrder.SaveConfigFile(this);
        }

        public void CrawlStart()
        {
            if (!m_IsRunning)
            {
                foreach (String url in m_CrawlOrder.SourceLinks)
                {
                    if (url != null && url != String.Empty)
                    {
                        m_CrawlUriManagement.Schedule(url, false, 0);
                    }
                }

                AddSpiders();
                m_IsRunning = true;
            }
        }

        /*
         * This function will return immediately
         * when all the spider stops, will trigger event: AllSpdiersStop CallBack
         */
        public void AsyncCrawlStop()
        {
            if (m_IsRunning)
            {
                m_CrawlOrder.m_SpiderNum = 0;
            }
        }

        public void CrawlStop()
        {
            if (m_IsRunning)
            {
                Int32 trying = 3;
                m_CrawlOrder.m_SpiderNum = 0;

                while (m_IsRunning && trying > 0)
                {
                    Thread.Sleep(500);
                    trying--;
                }

                XThreadPool.StopAll();
                m_IsRunning = false;

                for (Int32 i = 0; i < m_SpiderXisRunning.Count; i++)
                    m_SpiderXisRunning[i] = false;
            }
        }

        //Properties
        public Boolean IsRunning              { get { return m_IsRunning; } }
        public ProcessorChain UriProcessChain { get { return m_ProcessChain; } }
        public ObservableXubject Xbuject      { get { return m_Xbject; } }
        public Int32 AlreadyFinished          { get { return m_CrawlUriManagement.AlreadyProcessed; } }
        public CrawlOrder CrawlOrder          { get { return m_CrawlOrder; } }
        public Int32 MaxThreadNum             { get { return m_MaxThreadNum; } }
        public UriManagement UriManagement    { get { return m_CrawlUriManagement; } }

        public Int32 ShouldRunning
        {
            get { return m_CrawlOrder.m_SpiderNum; }
            set
            {
                m_CrawlOrder.m_SpiderNum = value;
                if (m_IsRunning)
                    AddSpiders();
            }
        }

        public void Release()
        {
            m_CrawlUriManagement.Release();
            lock(s_SyncRoot)
                m_SpiderXisRunning.Clear();
            m_RequestCollection.Clear();

            m_CrawlOrder.ReleaseProcessorInfo();
            m_Filters.Clear();

            Logger.Release();
            DnsCache.Release();
        }

        public void SuspendAllSpiders()
        {
            XThreadPool.SuspendAll();
        }

        public void ContinueAllSpiders()
        {
            XThreadPool.ResumeAll();
        }

        public Boolean AddProcessor(ProcessorInfo pinfo, ProcessorChain.ProcessorChainName ChainName)
        {
            try
            {
                Processor p = CreateProcessor(pinfo);
                if (p == null)
                    //the processor already existed...
                    return true;

                switch (ChainName)
                {
                    case ProcessorChain.ProcessorChainName.PreFetchChain:
                        {
                            m_PreFetchChain.AddProcessor(p);
                            p.m_Chain = m_PreFetchChain;
                        } return true;
                    case ProcessorChain.ProcessorChainName.FetchChain:
                        {
                            m_FetchChain.AddProcessor(p);
                            p.m_Chain = m_FetchChain;
                        } return true;
                    case ProcessorChain.ProcessorChainName.PostFetchChain:
                        {
                            m_PostFetchChain.AddProcessor(p);
                            p.m_Chain = m_PostFetchChain;
                        } return true;
                    case ProcessorChain.ProcessorChainName.ExtractChain:
                        {
                            m_ExtractChain.AddProcessor(p);
                            p.m_Chain = m_ExtractChain;
                        } return true;
                    case ProcessorChain.ProcessorChainName.StoreChain:
                        {
                            m_StoreChain.AddProcessor(p);
                            p.m_Chain = m_StoreChain;
                        } return true;
                    default: return false;
                }
            }
            catch (Exception)
            {
                return false;
            }
        }

        public Boolean AddFilter(ProcessorInfo filter)
        {
            try
            {
                if (GetFilterByName(filter.Name) != null)
                    return true;

                //may be loaded for many times ??
                Assembly assb = Assembly.LoadFrom(CrawlToolKits.GetAbsolutePath(s_Directory_Path, filter.m_Path));
                Type type_filter = assb.GetType(filter.m_ClassName);

                if (filter.Description == null)
                    filter.Description = "Nothing";

                Filter _filter = Activator.CreateInstance(type_filter, filter.m_Name, 
                                                                     filter.m_Description, 
                                                                     filter.m_ClassName, 
                                                                     filter.m_Path) as Filter;

                m_Filters.Add(_filter);

                return true;
            }
            catch (Exception)
            {
                return false;
            }
        }

        public void DeleteProcessor(String procname)
        {
            Processor proc = GetProcessorFromChain(procname);

            if (proc != null)
            {
                proc.m_Chain.DeleteProcessor(proc);
                proc.Filter = null;
            }

        }

        public void DeleteFilterFromProcessor(String procname)
        {
            Processor proc = GetProcessorFromChain(procname);

            if (proc != null)
            {
                proc.Filter = null;
            }
        }

        public void DeleteFilter(String filtername)
        {
            foreach (Processor proc in m_ProcessChain)
            {
                if (proc.Filter.Name == filtername)
                {
                    proc.Filter = null;
                }
            }
            foreach (Filter f in m_Filters)
            {
                if (f.Name == filtername)
                    m_Filters.Remove(f);
            }
        }

        public void DisableProcessor(String procname, Boolean disabled)
        {
            Processor proc = GetProcessorFromChain(procname);

            if (proc != null)
            {
                proc.Enabled = disabled;
            }
        }

        public Processor GetProcessorFromChain(String procname)
        {
            foreach (Processor proc in m_ProcessChain)
            {
                if (proc.Name == procname)
                {
                    return proc;
                }
            }

            return null;
        }

        public Filter GetFilterByName(String filtername)
        {
            foreach (Filter f in m_Filters)
            {
                if (f.Name == filtername)
                    return f;
            }
            return null;
        }

        public void DisableFilter(String filtername, Boolean disabled)
        {
            Filter f = GetFilterByName(filtername);
            if (f != null)
                f.Enabled = disabled;
        }

        protected Processor CreateProcessor(ProcessorInfo pinfo)
        {
            foreach (Processor proc in m_ProcessChain)
            {
                if (proc.Name == pinfo.Name)
                    return null;
            }

            Filter attfilter = null;
            Assembly assb = Assembly.LoadFrom(CrawlToolKits.GetAbsolutePath(s_Directory_Path, pinfo.m_Path));
            Type type_pinfo = assb.GetType(pinfo.m_ClassName);

            if ( pinfo.m_Attachedfilter != null && 
                 pinfo.m_Attachedfilter != String.Empty
               )
            {
                attfilter = GetFilterByName(pinfo.m_Attachedfilter);

                if (attfilter == null)
                    throw new ArgumentException("No target filter found in config.xml for Processor " + pinfo.m_ClassName);
            }

            if (pinfo.Description == null)
                pinfo.Description = "Nothing";

            Processor pro = Activator.CreateInstance(type_pinfo, pinfo.m_Name, 
                                                                 pinfo.m_Description, 
                                                                 pinfo.m_ClassName, 
                                                                 pinfo.m_Path, attfilter) as Processor;

            if (!pro.Inintialize())
                throw new Exception("Process " + pinfo.m_Name + " initial failed!");
            else
            {
                pro.m_Controller = this;
                return pro;
            }
        }

        protected void AddSpiders()
        {
            Int32 retry = 0x02;

            for (Int32 i = 0; i < m_CrawlOrder.m_SpiderNum; i++)
            {
                if (!m_SpiderXisRunning[i])
                {
                    CrawlSpider clsp = new CrawlSpider(this);
                    lock(s_SyncRoot)
                        m_SpiderXisRunning[i] = true;

                    while (retry > 0 && !XThreadPool.AssignNewTask(new RunTarget(clsp.XSpiderRun)))
                    {
                        Thread.Sleep(500);
                        retry--;
                    }
                    if (retry == 0)
                        throw new Exception("Can not run the spider immediately.");
                }
            }
        }

        private void CheckIfAllStops(Object sender, System.Timers.ElapsedEventArgs ee)
        {
            lock (s_SyncRoot)
            {
                if (!m_IsRunning)
                    return;

                foreach (Boolean b in m_SpiderXisRunning)
                {
                    if (b)
                        return;
                }
                //reach here means all the spider has stoped.

                m_IsRunning = false;
                m_CrawlUriManagement.Release();
                m_Xbject.OnAllSpidersStop();
            }
        }

        private Boolean CreateProcessorChain()
        {
            ProcessorChain chain = m_ProcessChain;
            while (chain != null)
            {
                chain.ClearProcessors();
                chain = chain.NextProcessorChain;
            }

            foreach (ProcessorInfo filter in m_CrawlOrder.m_Filters)
                if (!AddFilter(filter))
                    return false;

            foreach (ProcessorInfo prefetcher in m_CrawlOrder.m_PreFetchers)
                if (!AddProcessor(prefetcher, ProcessorChain.ProcessorChainName.PreFetchChain))
                    return false;

            foreach (ProcessorInfo fetcher in m_CrawlOrder.m_Fetchers)
                if (!AddProcessor(fetcher, ProcessorChain.ProcessorChainName.FetchChain))
                    return false;

            foreach (ProcessorInfo postfetcher in m_CrawlOrder.m_PostFetchers)
                if (!AddProcessor(postfetcher, ProcessorChain.ProcessorChainName.PostFetchChain))
                    return false;

            foreach (ProcessorInfo extractor in m_CrawlOrder.m_Extractors)
                if (!AddProcessor(extractor, ProcessorChain.ProcessorChainName.ExtractChain))
                    return false;

            foreach (ProcessorInfo store in m_CrawlOrder.m_Storages)
                if (!AddProcessor(store, ProcessorChain.ProcessorChainName.StoreChain))
                    return false;

            return true;
        }

        internal volatile List<Boolean> m_SpiderXisRunning;

        private  Int32 m_MaxThreadNum;

        internal Dictionary<Int32, XWebRequest>  m_RequestCollection;
        internal List<Filter> m_Filters;

        internal CrawlOrder     m_CrawlOrder;
        private  UriManagement  m_CrawlUriManagement;

        private volatile Boolean m_IsRunning;
        //
        private ObservableXubject m_Xbject;
        private System.Timers.Timer m_Timer;

        private ProcessorChain m_ProcessChain;
        private ProcessorChain m_PreFetchChain;
        private ProcessorChain m_FetchChain;
        private ProcessorChain m_PostFetchChain;
        private ProcessorChain m_ExtractChain;
        private ProcessorChain m_StoreChain;
    }
}
